# -*- coding:utf-8 -*-

import requests
import json
import time
import random


def jd_spilder(page=0):
    url = "https://sclub.jd.com/comment/productPageComments.action" \
          "?callback=fetchJSON_comment98vv4086&productId=7426552&score=0" \
          "&sortType=5&page=%s&pageSize=10&isShadowSku=0&rid=0&fold=1" % page
    request_headers = {"Referer": "https://item.jd.com/7426552.html", "User-Agent": "Mozilla/5.0"}

    try:
        res = requests.get(url, headers=request_headers)
        res.raise_for_status()
        comment = res.text[res.text.index("{"):-2]
        # print(comment)
        return comment
    except:
        print('爬取第' + str(page) + '页失败')





def batch_spilder(start_page, num):
    """批量爬取"""
    for i in range(start_page, num + 1):
        time.sleep(random.randint(2, 7))
        print('开始爬取第' + str(i) + '页')
        str_to_json(jd_spilder(i))


'''
转为json
提取评论内容 
写入文件
'''


def str_to_json(comment):
    if comment is None:
        print(comment)
        return
    json_str = json.loads(comment)
    comments = json_str['comments']
    with open('comments.txt', 'a+', encoding='utf-8') as file:
        for c in comments:
            # print(c['content'])
            file.write(c['content'])


if __name__ == '__main__':
    batch_spilder(501, 1000)
